from threading import Thread
from threading import Lock
from queue import Queue
import os
import requests
from UA_info import UA_list
import random
from bs4 import BeautifulSoup
from bs4.element import SoupStrainer
import time


class HostsSpider:
    def __init__(self):
        self.url = 'https://ipaddress.com/website/{}'
        self.url_queue = Queue()
        self.file_lock = Lock()
        with open('Github Address.txt', 'r') as file:
            self.address_list = file.readlines()  # 带'\n'
        if os.path.exists('hosts'):
            os.remove('hosts')
        self.hosts_file = open('hosts', 'w', newline='', encoding='utf-8')  # 记得.close()

    def init_url_queue(self):
        while self.address_list:
            current_address = self.address_list.pop(0).replace('\n', '')
            info = {
                'address': current_address,
                'url': self.url.format(current_address)
            }
            self.url_queue.put(info)

    def get(self):
        while not self.url_queue.empty():
            info = self.url_queue.get()
            print('正在爬取{}...'.format(info['address']))
            headers = {'User-Agent': random.choice(UA_list)}
            response = requests.get(url=info['url'], headers=headers)
            html = response.text
            self.parse(html, info)

    def parse(self, html, info):
        only_ul_tag = SoupStrainer('ul', class_='comma-separated')
        BS = BeautifulSoup(html, 'lxml', parse_only=only_ul_tag)
        host = BS.find('li').get_text()
        self.save(host, info)

    def save(self, host, info):
        if len(host) != 15:
            host += ' ' * (15 - len(host))
        self.file_lock.acquire()
        with open('hosts', 'a', newline='', encoding='utf-8') as file:
            file.write(host)
            file.write(' ')
            file.write(info['address'])
            file.write('\n')
        self.file_lock.release()

    def run(self):
        thread_list = []
        for i in range(36):
            new_thread = Thread(target=self.get)  # target的值是函数名，不带括号...
            new_thread.start()
            thread_list.append(new_thread)
        for old_thread in thread_list:
            old_thread.join()
        self.hosts_file.close()


if __name__ == '__main__':
    start = time.time()
    spider = HostsSpider()
    spider.init_url_queue()
    spider.run()
    end = time.time()
    print(f'所用时间：{end - start}')
